{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "from cot.stats import evaluation_as_table\n",
    "from cot.create import create_thoughtsource\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ts_100 = Collection.load_thoughtsource_100(load_pregenerated_cots=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading commonsense_qa...\n",
      "Loading med_qa...\n",
      "Loading medmc_qa...\n",
      "Loading medmc_qa...\n",
      "Loading open_book_qa...\n",
      "Loading strategy_qa...\n",
      "Loading worldtree...\n"
     ]
    }
   ],
   "source": [
    "ts_600 = create_thoughtsource(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading commonsense_qa...\n",
      "Loading med_qa...\n",
      "Loading medmc_qa...\n",
      "Loading medmc_qa...\n",
      "Loading open_book_qa...\n",
      "Loading strategy_qa...\n",
      "Loading worldtree...\n"
     ]
    }
   ],
   "source": [
    "ts_200 = create_thoughtsource(33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = Collection.load_thoughtsource_100()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_idx = ts.dump_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_600_idx = ts_600.dump_ids()\n",
    "ts_200_idx = ts_200.dump_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.seed(0)\n",
    "idx_1 = sorted(random.sample(range(0, 600), 50))\n",
    "random.seed(0)\n",
    "idx_2 = sorted(random.sample(range(0, 600), 200))\n",
    "len([item for item in idx_1 if item in idx_2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "random.seed(0)\n",
    "idx_1 = sorted(random.sample(range(0, 600), 50))\n",
    "\n",
    "random.seed(0)\n",
    "idx_2 = sorted(random.sample(range(0, 600), 200))\n",
    "\n",
    "common_elements = len([item for item in idx_1 if item in idx_2])\n",
    "\n",
    "print(common_elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "random.seed(0)\n",
    "sample1 = sorted(random.sample(range(0, 600), 100))\n",
    "\n",
    "random.seed(0)\n",
    "sample2 = sorted(random.sample(range(0, 600), 100))\n",
    "\n",
    "print(sample1 == sample2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indx = sorted(random.sample(range(0, 600), 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "394"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.seed(0)\n",
    "random.randint(0, 600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[97, 53, 5, 33, 65, 62, 51, 38, 61, 45]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.sample(range(0, 100), 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'idx' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/home/kon/CoT/ThoughtSource/notebooks/internal_documentation/konstantin_saves/item_selection_dataset_creation.ipynb Cell 8\u001b[0m in \u001b[0;36m3\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/kon/CoT/ThoughtSource/notebooks/internal_documentation/konstantin_saves/item_selection_dataset_creation.ipynb#Y106sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mrandom\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/kon/CoT/ThoughtSource/notebooks/internal_documentation/konstantin_saves/item_selection_dataset_creation.ipynb#Y106sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m random\u001b[39m.\u001b[39mseed(\u001b[39m0\u001b[39m)\n\u001b[0;32m----> <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/kon/CoT/ThoughtSource/notebooks/internal_documentation/konstantin_saves/item_selection_dataset_creation.ipynb#Y106sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m random_ids \u001b[39m=\u001b[39m random\u001b[39m.\u001b[39msample(idx, number_samples)\n\u001b[1;32m      <a href='vscode-notebook-cell://wsl%2Bubuntu-20.04/home/kon/CoT/ThoughtSource/notebooks/internal_documentation/konstantin_saves/item_selection_dataset_creation.ipynb#Y106sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m random_ids \u001b[39m=\u001b[39m \u001b[39msorted\u001b[39m(random_ids)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'idx' is not defined"
     ]
    }
   ],
   "source": [
    "import random\n",
    "random.seed(0)\n",
    "random.sample(range(0, 100), 10)\n",
    "random_ids = random.sample(idx, number_samples)\n",
    "random_ids = sorted(random_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(600, 198)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ts_600_idx), len(ts_200_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([item for item in ts_200_idx if item not in ts_600_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([item for item in ts_idx if item not in ts_600_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "423"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([item for item in ts_600_idx if item not in ts_200_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "open_600 = [item for item in ts_600_idx if item.startswith(\"open\")]\n",
    "open_200 = [item for item in ts_200_idx if item.startswith(\"open\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([item for item in open_600 if item not in open_200])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['open_book_qa_test_494', 'open_book_qa_test_495']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[item for item in open_200 if item not in open_600]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset open_book_qa_dataset (/home/kon/.cache/huggingface/datasets/open_book_qa_dataset/thoughtsource/1.0.0/b1163057b19c0b0be4f0f1104103cb49d35de01177baec5250adb5246116272f)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading open_book_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfcc5df8c7874ca191b254cd1ef3e416",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "caf21e85cd0d4c08a9275b9cccfba02c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4957 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1380a604b21477fac0490f252186d0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98806d3d17b64abe90dfb720cc1b1655",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset open_book_qa_dataset (/home/kon/.cache/huggingface/datasets/open_book_qa_dataset/thoughtsource/1.0.0/b1163057b19c0b0be4f0f1104103cb49d35de01177baec5250adb5246116272f)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading open_book_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "550120b54e98444aa1ac0b10547e3996",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5d6d0159dfa4f6b92ac265203ead523",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4957 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d148f193bf6042c39953c0490a6493d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e715b2cc55e42c2b9a49e0a3daeed50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "open_direct_600 = Collection(\"open_book_qa\", load_pregenerated_cots=False)\n",
    "open_direct_200 = Collection(\"open_book_qa\", load_pregenerated_cots=False)\n",
    "open_direct_600 = open_direct_600.select(\"test\", 100).dump_ids()\n",
    "open_direct_200 = open_direct_200.select(\"test\", 33).dump_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name   | Train   | Valid   | Test   |\n",
       "|--------|---------|---------|--------|\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'open_book_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp', 'worldtree']"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Collection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset worldtree_dataset (/home/kon/.cache/huggingface/datasets/worldtree_dataset/thoughtsource/1.0.0/4ec0cd827b41f05891af9a27bf461fecd407e2fe7c1beebfed1eb00193c2cd52)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d219ce0e731a488481a56aa807ad9d49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44d91b5761b2499ebb2515377d47e7e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2207 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ca24b23d95e4ae594337e05f7e08170",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1664 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "536c4dd705e94a02b8a64d0f47006fff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/496 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset worldtree_dataset (/home/kon/.cache/huggingface/datasets/worldtree_dataset/thoughtsource/1.0.0/4ec0cd827b41f05891af9a27bf461fecd407e2fe7c1beebfed1eb00193c2cd52)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58bc41d3811741869f4a4d0d11394aa1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6365977fd7c44ea89bdf294ed1a46f62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2207 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "687afc1ccb454c1ca5ca1026bf051ffa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1664 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21370d4dded549e5adb3fe5948095a16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/496 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "open_direct_600 = Collection(\"worldtree\", load_pregenerated_cots=False)\n",
    "open_direct_200 = Collection(\"worldtree\", load_pregenerated_cots=False)\n",
    "open_direct_600 = open_direct_600.select(\"test\", 100).dump_ids()\n",
    "open_direct_200 = open_direct_200.select(\"test\", 33).dump_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for name in ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'open_book_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp', 'worldtree']:\n",
    "    try:\n",
    "        open_direct_600 = Collection(name, load_pregenerated_cots=False)\n",
    "        open_direct_200 = Collection(name, load_pregenerated_cots=False)\n",
    "        open_direct_600 = open_direct_600.select(\"test\", 100).dump_ids()\n",
    "        open_direct_200 = open_direct_200.select(\"test\", 33).dump_ids()\n",
    "        print(name, len([item for item in open_direct_200 if item not in open_direct_600]))\n",
    "    except Exception as e:\n",
    "        print(name, e)\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 33)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(open_direct_600), len(open_direct_200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[item for item in open_direct_200 if item not in open_direct_600]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[item for item in open_direct_200 if item not in open_200]\n",
    "[item for item in open_200 if item not in open_direct_200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([item for item in ts_200_idx if item.startswith(\"open\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['efa0e92a-b11b-4c1c-a97d-8b4409430caa',\n",
       " '75875e90-359b-410d-a777-4cd41003fe93',\n",
       " 'e4e65063-c1b6-43c7-9dbc-f18bffde95aa',\n",
       " '15f7993b-1b86-400a-83c1-a91417ab640c',\n",
       " '2fd85795-3aca-4bd2-b367-256d33655e91',\n",
       " '98893c1c-a979-476e-8b82-ef3bd2a02da7',\n",
       " 'eaa8c755-dc9b-4a75-ac66-4664ee83567a',\n",
       " 'cc50db92-b976-499e-b7ef-2d8c27458043',\n",
       " '64dbc15a-1052-4e21-8f02-e83da8a1e157',\n",
       " '5c6f357f-dbae-4930-9dbc-e3f3a4d064b1',\n",
       " 'b55fd5c5-4bb5-41d9-b78f-0932ac3854e6',\n",
       " '138f2721-a8f6-4456-a302-e63411415f00',\n",
       " '4f393b71-3955-4db3-a376-99af5015b825',\n",
       " 'b0b23ecd-6076-476c-935b-a6384c8474ff',\n",
       " '4d1e7661-fffc-42d0-8bed-1e4ecc040fe8',\n",
       " 'fffb00bf-d5e6-4975-9299-12beacacb8ad',\n",
       " 'db180b6d-8b4e-487e-a47d-7c554c8dad2e',\n",
       " '95deaff5-dc0b-42a7-b257-573f3de4c88d',\n",
       " '4566acfa-b637-4b16-9602-b7b88e4012d2',\n",
       " 'open_book_qa_test_494',\n",
       " 'open_book_qa_test_495']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[item for item in ts_200_idx if item not in ts_600_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import loadtxt, savetxt\n",
    "ts_100_ids = loadtxt(\"ts_100_ids.txt\", dtype=str, comments=\"#\", delimiter=\",\", unpack=False)\n",
    "ts_167_ids = loadtxt(\"ts_167_ids.txt\", dtype=str, comments=\"#\", delimiter=\",\", unpack=False)\n",
    "ts_200_ids = loadtxt(\"ts_200_ids.txt\", dtype=str, comments=\"#\", delimiter=\",\", unpack=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1002, 600, 198)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ts_167_ids), len(ts_100_ids), len(ts_200_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(402, 402)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# assert that difference of length of lists are right\n",
    "len(ts_167_ids) - len(ts_100_ids), len([item for item in ts_167_ids if item not in ts_100_ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(804, 822)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ts_167_ids) - len(ts_200_ids), len([item for item in ts_167_ids if item not in ts_200_ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(402, 423)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ts_100_ids) - len(ts_200_ids), len([item for item in ts_100_ids if item not in ts_200_ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff_1 = [item for item in ts_100_ids if item not in ts_200_ids]\n",
    "diff_2 = [item for item in ts_167_ids if item not in ts_100_ids]\n",
    "diff_final = [item for item in diff_2 if item in diff_1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['efa0e92a-b11b-4c1c-a97d-8b4409430caa',\n",
       " '75875e90-359b-410d-a777-4cd41003fe93',\n",
       " 'e4e65063-c1b6-43c7-9dbc-f18bffde95aa',\n",
       " '15f7993b-1b86-400a-83c1-a91417ab640c',\n",
       " '2fd85795-3aca-4bd2-b367-256d33655e91',\n",
       " '98893c1c-a979-476e-8b82-ef3bd2a02da7',\n",
       " 'eaa8c755-dc9b-4a75-ac66-4664ee83567a',\n",
       " 'cc50db92-b976-499e-b7ef-2d8c27458043',\n",
       " '64dbc15a-1052-4e21-8f02-e83da8a1e157',\n",
       " '5c6f357f-dbae-4930-9dbc-e3f3a4d064b1',\n",
       " 'b55fd5c5-4bb5-41d9-b78f-0932ac3854e6',\n",
       " '138f2721-a8f6-4456-a302-e63411415f00',\n",
       " '4f393b71-3955-4db3-a376-99af5015b825',\n",
       " 'b0b23ecd-6076-476c-935b-a6384c8474ff',\n",
       " '4d1e7661-fffc-42d0-8bed-1e4ecc040fe8',\n",
       " 'fffb00bf-d5e6-4975-9299-12beacacb8ad',\n",
       " 'db180b6d-8b4e-487e-a47d-7c554c8dad2e',\n",
       " '95deaff5-dc0b-42a7-b257-573f3de4c88d',\n",
       " '4566acfa-b637-4b16-9602-b7b88e4012d2',\n",
       " 'open_book_qa_test_494',\n",
       " 'open_book_qa_test_495']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[item for item in ts_200_ids if item not in ts_100_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "402"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([item for item in ts_167_ids if item not in ts_100_ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = [item for item in ts_100_ids if item not in ts_167_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/kon/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# ts_167 = create_thoughtsource(167, load_pregenerated_cots=False)\n",
    "ts_167 = Collection.from_json(\"ts_167_empty.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading commonsense_qa...\n",
      "Loading med_qa...\n",
      "Loading medmc_qa...\n",
      "Loading medmc_qa...\n",
      "Loading open_book_qa...\n",
      "Loading strategy_qa...\n",
      "Loading worldtree...\n"
     ]
    }
   ],
   "source": [
    "ts_200 = create_thoughtsource(33, load_pregenerated_cots=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_200.dump(\"ts_200.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_200.dump_ids(\"ts_200_ids.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_167.dump_ids(\"ts_167_ids.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ts_100_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for -: 'list' and 'list'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39mlen\u001b[39m(\u001b[39mlist\u001b[39;49m(ts_167_ids) \u001b[39m-\u001b[39;49m \u001b[39mlist\u001b[39;49m(ts_100_ids))\n",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for -: 'list' and 'list'"
     ]
    }
   ],
   "source": [
    "len(list(ts_167_ids) - list(ts_100_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a boolean mask for elements in array1 that are not in array2\n",
    "mask = np.isin(ts_167_ids, ts_100_ids, invert=True)\n",
    "\n",
    "# Apply the mask to array1 to get the result with original order\n",
    "result = ts_167_ids[mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "67.0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result)/6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 167     | -      |\n",
       "| med_qa         | -       | -       | 167    |\n",
       "| medmc_qa       | -       | 167     | -      |\n",
       "| open_book_qa   | -       | -       | 167    |\n",
       "| strategy_qa    | 167     | -       | -      |\n",
       "| worldtree      | -       | -       | 167    |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_167"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 100     | -      |\n",
       "| med_qa         | -       | -       | 100    |\n",
       "| medmc_qa       | -       | 100     | -      |\n",
       "| open_book_qa   | -       | -       | 100    |\n",
       "| strategy_qa    | 100     | -       | -      |\n",
       "| worldtree      | -       | -       | 100    |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sometimes this works and sometimes it throws an error\n",
    "# if it throws an error, just run it again with ts_100.filter() and then again with ts_167.filter()\n",
    "ts_67 = ts_167.filter(lambda ex: ex['id'] not in ts_100_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 67      | -      |\n",
       "| med_qa         | -       | -       | 67     |\n",
       "| medmc_qa       | -       | 67      | -      |\n",
       "| open_book_qa   | -       | -       | 67     |\n",
       "| strategy_qa    | 67      | -       | -      |\n",
       "| worldtree      | -       | -       | 67     |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_67"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_67.dump(\"ts_67_empty\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/kon/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# thoughtsource 100 index renaming\n",
    "ts_100 = Collection.load_thoughtsource_100(load_pregenerated_cots=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100.select_generated_cots(author=\"thoughtsource\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kon/work/ThoughtSource/libs/cot/cot/stats.py:406: PerformanceWarning: indexing past lexsort depth may impact performance.\n",
      "  df.loc[dataset, (instruction + \"_\" + cot_trigger, model)] = v\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_a9ae5_row0_col1, #T_a9ae5_row1_col15, #T_a9ae5_row2_col15, #T_a9ae5_row3_col15, #T_a9ae5_row4_col15, #T_a9ae5_row5_col3, #T_a9ae5_row6_col15 {\n",
       "  font-weight: bold;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_a9ae5\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_a9ae5_level0_col0\" class=\"col_heading level0 col0\" colspan=\"6\">None_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col6\" class=\"col_heading level0 col6\" colspan=\"5\">None_kojima-01</th>\n",
       "      <th id=\"T_a9ae5_level0_col11\" class=\"col_heading level0 col11\" >None_kojima-03</th>\n",
       "      <th id=\"T_a9ae5_level0_col12\" class=\"col_heading level0 col12\" >None_kojima-09</th>\n",
       "      <th id=\"T_a9ae5_level0_col13\" class=\"col_heading level0 col13\" colspan=\"3\">None_zhou-01</th>\n",
       "      <th id=\"T_a9ae5_level0_col16\" class=\"col_heading level0 col16\" >qa-01_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col17\" class=\"col_heading level0 col17\" >qa-05_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col18\" class=\"col_heading level0 col18\" >qa-08_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col19\" class=\"col_heading level0 col19\" >qa-09_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col20\" class=\"col_heading level0 col20\" >qa-10_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col21\" class=\"col_heading level0 col21\" >qa-12_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col22\" class=\"col_heading level0 col22\" >qa-13_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col23\" class=\"col_heading level0 col23\" >qa-16_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col24\" class=\"col_heading level0 col24\" >qa-17_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col25\" class=\"col_heading level0 col25\" >zhou-01-ins_None</th>\n",
       "      <th id=\"T_a9ae5_level0_col26\" class=\"col_heading level0 col26\" >zhou-01-ins_zhou-01</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"blank level1\" >&nbsp;</th>\n",
       "      <th id=\"T_a9ae5_level1_col0\" class=\"col_heading level1 col0\" >command-xlarge-nightly</th>\n",
       "      <th id=\"T_a9ae5_level1_col1\" class=\"col_heading level1 col1\" >flan-T5-xxl</th>\n",
       "      <th id=\"T_a9ae5_level1_col2\" class=\"col_heading level1 col2\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col3\" class=\"col_heading level1 col3\" >gpt-4</th>\n",
       "      <th id=\"T_a9ae5_level1_col4\" class=\"col_heading level1 col4\" >text-davinci-002</th>\n",
       "      <th id=\"T_a9ae5_level1_col5\" class=\"col_heading level1 col5\" >text-davinci-003</th>\n",
       "      <th id=\"T_a9ae5_level1_col6\" class=\"col_heading level1 col6\" >command-xlarge-nightly</th>\n",
       "      <th id=\"T_a9ae5_level1_col7\" class=\"col_heading level1 col7\" >flan-T5-xxl</th>\n",
       "      <th id=\"T_a9ae5_level1_col8\" class=\"col_heading level1 col8\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col9\" class=\"col_heading level1 col9\" >text-davinci-002</th>\n",
       "      <th id=\"T_a9ae5_level1_col10\" class=\"col_heading level1 col10\" >text-davinci-003</th>\n",
       "      <th id=\"T_a9ae5_level1_col11\" class=\"col_heading level1 col11\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col12\" class=\"col_heading level1 col12\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col13\" class=\"col_heading level1 col13\" >flan-T5-xxl</th>\n",
       "      <th id=\"T_a9ae5_level1_col14\" class=\"col_heading level1 col14\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col15\" class=\"col_heading level1 col15\" >gpt-4</th>\n",
       "      <th id=\"T_a9ae5_level1_col16\" class=\"col_heading level1 col16\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col17\" class=\"col_heading level1 col17\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col18\" class=\"col_heading level1 col18\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col19\" class=\"col_heading level1 col19\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col20\" class=\"col_heading level1 col20\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col21\" class=\"col_heading level1 col21\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col22\" class=\"col_heading level1 col22\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col23\" class=\"col_heading level1 col23\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col24\" class=\"col_heading level1 col24\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col25\" class=\"col_heading level1 col25\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_a9ae5_level1_col26\" class=\"col_heading level1 col26\" >gpt-3.5-turbo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_a9ae5_level0_row0\" class=\"row_heading level0 row0\" >commonsense_qa</th>\n",
       "      <td id=\"T_a9ae5_row0_col0\" class=\"data row0 col0\" >0.51</td>\n",
       "      <td id=\"T_a9ae5_row0_col1\" class=\"data row0 col1\" >0.87</td>\n",
       "      <td id=\"T_a9ae5_row0_col2\" class=\"data row0 col2\" >0.72</td>\n",
       "      <td id=\"T_a9ae5_row0_col3\" class=\"data row0 col3\" >0.75</td>\n",
       "      <td id=\"T_a9ae5_row0_col4\" class=\"data row0 col4\" >0.76</td>\n",
       "      <td id=\"T_a9ae5_row0_col5\" class=\"data row0 col5\" >0.72</td>\n",
       "      <td id=\"T_a9ae5_row0_col6\" class=\"data row0 col6\" >0.53</td>\n",
       "      <td id=\"T_a9ae5_row0_col7\" class=\"data row0 col7\" >0.81</td>\n",
       "      <td id=\"T_a9ae5_row0_col8\" class=\"data row0 col8\" >0.67</td>\n",
       "      <td id=\"T_a9ae5_row0_col9\" class=\"data row0 col9\" >0.62</td>\n",
       "      <td id=\"T_a9ae5_row0_col10\" class=\"data row0 col10\" >0.65</td>\n",
       "      <td id=\"T_a9ae5_row0_col11\" class=\"data row0 col11\" >0.63</td>\n",
       "      <td id=\"T_a9ae5_row0_col12\" class=\"data row0 col12\" >0.70</td>\n",
       "      <td id=\"T_a9ae5_row0_col13\" class=\"data row0 col13\" >0.83</td>\n",
       "      <td id=\"T_a9ae5_row0_col14\" class=\"data row0 col14\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row0_col15\" class=\"data row0 col15\" >0.72</td>\n",
       "      <td id=\"T_a9ae5_row0_col16\" class=\"data row0 col16\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row0_col17\" class=\"data row0 col17\" >0.69</td>\n",
       "      <td id=\"T_a9ae5_row0_col18\" class=\"data row0 col18\" >0.62</td>\n",
       "      <td id=\"T_a9ae5_row0_col19\" class=\"data row0 col19\" >0.64</td>\n",
       "      <td id=\"T_a9ae5_row0_col20\" class=\"data row0 col20\" >0.68</td>\n",
       "      <td id=\"T_a9ae5_row0_col21\" class=\"data row0 col21\" >0.63</td>\n",
       "      <td id=\"T_a9ae5_row0_col22\" class=\"data row0 col22\" >0.61</td>\n",
       "      <td id=\"T_a9ae5_row0_col23\" class=\"data row0 col23\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row0_col24\" class=\"data row0 col24\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row0_col25\" class=\"data row0 col25\" >0.72</td>\n",
       "      <td id=\"T_a9ae5_row0_col26\" class=\"data row0 col26\" >0.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a9ae5_level0_row1\" class=\"row_heading level0 row1\" >med_qa</th>\n",
       "      <td id=\"T_a9ae5_row1_col0\" class=\"data row1 col0\" >0.23</td>\n",
       "      <td id=\"T_a9ae5_row1_col1\" class=\"data row1 col1\" >0.32</td>\n",
       "      <td id=\"T_a9ae5_row1_col2\" class=\"data row1 col2\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row1_col3\" class=\"data row1 col3\" >0.73</td>\n",
       "      <td id=\"T_a9ae5_row1_col4\" class=\"data row1 col4\" >0.41</td>\n",
       "      <td id=\"T_a9ae5_row1_col5\" class=\"data row1 col5\" >0.43</td>\n",
       "      <td id=\"T_a9ae5_row1_col6\" class=\"data row1 col6\" >0.31</td>\n",
       "      <td id=\"T_a9ae5_row1_col7\" class=\"data row1 col7\" >0.34</td>\n",
       "      <td id=\"T_a9ae5_row1_col8\" class=\"data row1 col8\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row1_col9\" class=\"data row1 col9\" >0.34</td>\n",
       "      <td id=\"T_a9ae5_row1_col10\" class=\"data row1 col10\" >0.43</td>\n",
       "      <td id=\"T_a9ae5_row1_col11\" class=\"data row1 col11\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row1_col12\" class=\"data row1 col12\" >0.51</td>\n",
       "      <td id=\"T_a9ae5_row1_col13\" class=\"data row1 col13\" >0.27</td>\n",
       "      <td id=\"T_a9ae5_row1_col14\" class=\"data row1 col14\" >0.65</td>\n",
       "      <td id=\"T_a9ae5_row1_col15\" class=\"data row1 col15\" >0.76</td>\n",
       "      <td id=\"T_a9ae5_row1_col16\" class=\"data row1 col16\" >0.54</td>\n",
       "      <td id=\"T_a9ae5_row1_col17\" class=\"data row1 col17\" >0.53</td>\n",
       "      <td id=\"T_a9ae5_row1_col18\" class=\"data row1 col18\" >0.46</td>\n",
       "      <td id=\"T_a9ae5_row1_col19\" class=\"data row1 col19\" >0.55</td>\n",
       "      <td id=\"T_a9ae5_row1_col20\" class=\"data row1 col20\" >0.56</td>\n",
       "      <td id=\"T_a9ae5_row1_col21\" class=\"data row1 col21\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row1_col22\" class=\"data row1 col22\" >0.49</td>\n",
       "      <td id=\"T_a9ae5_row1_col23\" class=\"data row1 col23\" >0.60</td>\n",
       "      <td id=\"T_a9ae5_row1_col24\" class=\"data row1 col24\" >0.56</td>\n",
       "      <td id=\"T_a9ae5_row1_col25\" class=\"data row1 col25\" >0.54</td>\n",
       "      <td id=\"T_a9ae5_row1_col26\" class=\"data row1 col26\" >0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a9ae5_level0_row2\" class=\"row_heading level0 row2\" >medmc_qa</th>\n",
       "      <td id=\"T_a9ae5_row2_col0\" class=\"data row2 col0\" >0.25</td>\n",
       "      <td id=\"T_a9ae5_row2_col1\" class=\"data row2 col1\" >0.34</td>\n",
       "      <td id=\"T_a9ae5_row2_col2\" class=\"data row2 col2\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row2_col3\" class=\"data row2 col3\" >0.69</td>\n",
       "      <td id=\"T_a9ae5_row2_col4\" class=\"data row2 col4\" >0.34</td>\n",
       "      <td id=\"T_a9ae5_row2_col5\" class=\"data row2 col5\" >0.40</td>\n",
       "      <td id=\"T_a9ae5_row2_col6\" class=\"data row2 col6\" >0.22</td>\n",
       "      <td id=\"T_a9ae5_row2_col7\" class=\"data row2 col7\" >0.35</td>\n",
       "      <td id=\"T_a9ae5_row2_col8\" class=\"data row2 col8\" >0.47</td>\n",
       "      <td id=\"T_a9ae5_row2_col9\" class=\"data row2 col9\" >0.34</td>\n",
       "      <td id=\"T_a9ae5_row2_col10\" class=\"data row2 col10\" >0.36</td>\n",
       "      <td id=\"T_a9ae5_row2_col11\" class=\"data row2 col11\" >0.50</td>\n",
       "      <td id=\"T_a9ae5_row2_col12\" class=\"data row2 col12\" >0.50</td>\n",
       "      <td id=\"T_a9ae5_row2_col13\" class=\"data row2 col13\" >0.31</td>\n",
       "      <td id=\"T_a9ae5_row2_col14\" class=\"data row2 col14\" >0.48</td>\n",
       "      <td id=\"T_a9ae5_row2_col15\" class=\"data row2 col15\" >0.70</td>\n",
       "      <td id=\"T_a9ae5_row2_col16\" class=\"data row2 col16\" >0.47</td>\n",
       "      <td id=\"T_a9ae5_row2_col17\" class=\"data row2 col17\" >0.42</td>\n",
       "      <td id=\"T_a9ae5_row2_col18\" class=\"data row2 col18\" >0.45</td>\n",
       "      <td id=\"T_a9ae5_row2_col19\" class=\"data row2 col19\" >0.47</td>\n",
       "      <td id=\"T_a9ae5_row2_col20\" class=\"data row2 col20\" >0.49</td>\n",
       "      <td id=\"T_a9ae5_row2_col21\" class=\"data row2 col21\" >0.53</td>\n",
       "      <td id=\"T_a9ae5_row2_col22\" class=\"data row2 col22\" >0.41</td>\n",
       "      <td id=\"T_a9ae5_row2_col23\" class=\"data row2 col23\" >0.48</td>\n",
       "      <td id=\"T_a9ae5_row2_col24\" class=\"data row2 col24\" >0.53</td>\n",
       "      <td id=\"T_a9ae5_row2_col25\" class=\"data row2 col25\" >0.44</td>\n",
       "      <td id=\"T_a9ae5_row2_col26\" class=\"data row2 col26\" >0.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a9ae5_level0_row3\" class=\"row_heading level0 row3\" >open_book_qa</th>\n",
       "      <td id=\"T_a9ae5_row3_col0\" class=\"data row3 col0\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row3_col1\" class=\"data row3 col1\" >0.82</td>\n",
       "      <td id=\"T_a9ae5_row3_col2\" class=\"data row3 col2\" >0.77</td>\n",
       "      <td id=\"T_a9ae5_row3_col3\" class=\"data row3 col3\" >0.92</td>\n",
       "      <td id=\"T_a9ae5_row3_col4\" class=\"data row3 col4\" >0.67</td>\n",
       "      <td id=\"T_a9ae5_row3_col5\" class=\"data row3 col5\" >0.70</td>\n",
       "      <td id=\"T_a9ae5_row3_col6\" class=\"data row3 col6\" >0.38</td>\n",
       "      <td id=\"T_a9ae5_row3_col7\" class=\"data row3 col7\" >0.79</td>\n",
       "      <td id=\"T_a9ae5_row3_col8\" class=\"data row3 col8\" >0.77</td>\n",
       "      <td id=\"T_a9ae5_row3_col9\" class=\"data row3 col9\" >0.57</td>\n",
       "      <td id=\"T_a9ae5_row3_col10\" class=\"data row3 col10\" >0.67</td>\n",
       "      <td id=\"T_a9ae5_row3_col11\" class=\"data row3 col11\" >0.73</td>\n",
       "      <td id=\"T_a9ae5_row3_col12\" class=\"data row3 col12\" >0.73</td>\n",
       "      <td id=\"T_a9ae5_row3_col13\" class=\"data row3 col13\" >0.81</td>\n",
       "      <td id=\"T_a9ae5_row3_col14\" class=\"data row3 col14\" >0.81</td>\n",
       "      <td id=\"T_a9ae5_row3_col15\" class=\"data row3 col15\" >0.95</td>\n",
       "      <td id=\"T_a9ae5_row3_col16\" class=\"data row3 col16\" >0.73</td>\n",
       "      <td id=\"T_a9ae5_row3_col17\" class=\"data row3 col17\" >0.65</td>\n",
       "      <td id=\"T_a9ae5_row3_col18\" class=\"data row3 col18\" >0.73</td>\n",
       "      <td id=\"T_a9ae5_row3_col19\" class=\"data row3 col19\" >0.71</td>\n",
       "      <td id=\"T_a9ae5_row3_col20\" class=\"data row3 col20\" >0.73</td>\n",
       "      <td id=\"T_a9ae5_row3_col21\" class=\"data row3 col21\" >0.80</td>\n",
       "      <td id=\"T_a9ae5_row3_col22\" class=\"data row3 col22\" >0.72</td>\n",
       "      <td id=\"T_a9ae5_row3_col23\" class=\"data row3 col23\" >0.69</td>\n",
       "      <td id=\"T_a9ae5_row3_col24\" class=\"data row3 col24\" >0.69</td>\n",
       "      <td id=\"T_a9ae5_row3_col25\" class=\"data row3 col25\" >0.76</td>\n",
       "      <td id=\"T_a9ae5_row3_col26\" class=\"data row3 col26\" >0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a9ae5_level0_row4\" class=\"row_heading level0 row4\" >strategy_qa</th>\n",
       "      <td id=\"T_a9ae5_row4_col0\" class=\"data row4 col0\" >0.52</td>\n",
       "      <td id=\"T_a9ae5_row4_col1\" class=\"data row4 col1\" >0.61</td>\n",
       "      <td id=\"T_a9ae5_row4_col2\" class=\"data row4 col2\" >0.57</td>\n",
       "      <td id=\"T_a9ae5_row4_col3\" class=\"data row4 col3\" >0.71</td>\n",
       "      <td id=\"T_a9ae5_row4_col4\" class=\"data row4 col4\" >0.36</td>\n",
       "      <td id=\"T_a9ae5_row4_col5\" class=\"data row4 col5\" >0.53</td>\n",
       "      <td id=\"T_a9ae5_row4_col6\" class=\"data row4 col6\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row4_col7\" class=\"data row4 col7\" >0.69</td>\n",
       "      <td id=\"T_a9ae5_row4_col8\" class=\"data row4 col8\" >0.56</td>\n",
       "      <td id=\"T_a9ae5_row4_col9\" class=\"data row4 col9\" >0.38</td>\n",
       "      <td id=\"T_a9ae5_row4_col10\" class=\"data row4 col10\" >0.54</td>\n",
       "      <td id=\"T_a9ae5_row4_col11\" class=\"data row4 col11\" >0.52</td>\n",
       "      <td id=\"T_a9ae5_row4_col12\" class=\"data row4 col12\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row4_col13\" class=\"data row4 col13\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row4_col14\" class=\"data row4 col14\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row4_col15\" class=\"data row4 col15\" >0.80</td>\n",
       "      <td id=\"T_a9ae5_row4_col16\" class=\"data row4 col16\" >0.44</td>\n",
       "      <td id=\"T_a9ae5_row4_col17\" class=\"data row4 col17\" >0.43</td>\n",
       "      <td id=\"T_a9ae5_row4_col18\" class=\"data row4 col18\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row4_col19\" class=\"data row4 col19\" >0.62</td>\n",
       "      <td id=\"T_a9ae5_row4_col20\" class=\"data row4 col20\" >0.56</td>\n",
       "      <td id=\"T_a9ae5_row4_col21\" class=\"data row4 col21\" >0.50</td>\n",
       "      <td id=\"T_a9ae5_row4_col22\" class=\"data row4 col22\" >0.64</td>\n",
       "      <td id=\"T_a9ae5_row4_col23\" class=\"data row4 col23\" >0.63</td>\n",
       "      <td id=\"T_a9ae5_row4_col24\" class=\"data row4 col24\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row4_col25\" class=\"data row4 col25\" >0.52</td>\n",
       "      <td id=\"T_a9ae5_row4_col26\" class=\"data row4 col26\" >0.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a9ae5_level0_row5\" class=\"row_heading level0 row5\" >worldtree</th>\n",
       "      <td id=\"T_a9ae5_row5_col0\" class=\"data row5 col0\" >0.63</td>\n",
       "      <td id=\"T_a9ae5_row5_col1\" class=\"data row5 col1\" >0.85</td>\n",
       "      <td id=\"T_a9ae5_row5_col2\" class=\"data row5 col2\" >0.96</td>\n",
       "      <td id=\"T_a9ae5_row5_col3\" class=\"data row5 col3\" >0.99</td>\n",
       "      <td id=\"T_a9ae5_row5_col4\" class=\"data row5 col4\" >0.88</td>\n",
       "      <td id=\"T_a9ae5_row5_col5\" class=\"data row5 col5\" >0.91</td>\n",
       "      <td id=\"T_a9ae5_row5_col6\" class=\"data row5 col6\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row5_col7\" class=\"data row5 col7\" >0.80</td>\n",
       "      <td id=\"T_a9ae5_row5_col8\" class=\"data row5 col8\" >0.93</td>\n",
       "      <td id=\"T_a9ae5_row5_col9\" class=\"data row5 col9\" >0.78</td>\n",
       "      <td id=\"T_a9ae5_row5_col10\" class=\"data row5 col10\" >0.89</td>\n",
       "      <td id=\"T_a9ae5_row5_col11\" class=\"data row5 col11\" >0.95</td>\n",
       "      <td id=\"T_a9ae5_row5_col12\" class=\"data row5 col12\" >0.95</td>\n",
       "      <td id=\"T_a9ae5_row5_col13\" class=\"data row5 col13\" >0.83</td>\n",
       "      <td id=\"T_a9ae5_row5_col14\" class=\"data row5 col14\" >0.92</td>\n",
       "      <td id=\"T_a9ae5_row5_col15\" class=\"data row5 col15\" >0.99</td>\n",
       "      <td id=\"T_a9ae5_row5_col16\" class=\"data row5 col16\" >0.95</td>\n",
       "      <td id=\"T_a9ae5_row5_col17\" class=\"data row5 col17\" >0.74</td>\n",
       "      <td id=\"T_a9ae5_row5_col18\" class=\"data row5 col18\" >0.92</td>\n",
       "      <td id=\"T_a9ae5_row5_col19\" class=\"data row5 col19\" >0.91</td>\n",
       "      <td id=\"T_a9ae5_row5_col20\" class=\"data row5 col20\" >0.95</td>\n",
       "      <td id=\"T_a9ae5_row5_col21\" class=\"data row5 col21\" >0.92</td>\n",
       "      <td id=\"T_a9ae5_row5_col22\" class=\"data row5 col22\" >0.96</td>\n",
       "      <td id=\"T_a9ae5_row5_col23\" class=\"data row5 col23\" >0.91</td>\n",
       "      <td id=\"T_a9ae5_row5_col24\" class=\"data row5 col24\" >0.92</td>\n",
       "      <td id=\"T_a9ae5_row5_col25\" class=\"data row5 col25\" >0.96</td>\n",
       "      <td id=\"T_a9ae5_row5_col26\" class=\"data row5 col26\" >0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_a9ae5_level0_row6\" class=\"row_heading level0 row6\" >Average</th>\n",
       "      <td id=\"T_a9ae5_row6_col0\" class=\"data row6 col0\" >0.46</td>\n",
       "      <td id=\"T_a9ae5_row6_col1\" class=\"data row6 col1\" >0.64</td>\n",
       "      <td id=\"T_a9ae5_row6_col2\" class=\"data row6 col2\" >0.70</td>\n",
       "      <td id=\"T_a9ae5_row6_col3\" class=\"data row6 col3\" >0.80</td>\n",
       "      <td id=\"T_a9ae5_row6_col4\" class=\"data row6 col4\" >0.57</td>\n",
       "      <td id=\"T_a9ae5_row6_col5\" class=\"data row6 col5\" >0.62</td>\n",
       "      <td id=\"T_a9ae5_row6_col6\" class=\"data row6 col6\" >0.44</td>\n",
       "      <td id=\"T_a9ae5_row6_col7\" class=\"data row6 col7\" >0.63</td>\n",
       "      <td id=\"T_a9ae5_row6_col8\" class=\"data row6 col8\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row6_col9\" class=\"data row6 col9\" >0.50</td>\n",
       "      <td id=\"T_a9ae5_row6_col10\" class=\"data row6 col10\" >0.59</td>\n",
       "      <td id=\"T_a9ae5_row6_col11\" class=\"data row6 col11\" >0.65</td>\n",
       "      <td id=\"T_a9ae5_row6_col12\" class=\"data row6 col12\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row6_col13\" class=\"data row6 col13\" >0.61</td>\n",
       "      <td id=\"T_a9ae5_row6_col14\" class=\"data row6 col14\" >0.68</td>\n",
       "      <td id=\"T_a9ae5_row6_col15\" class=\"data row6 col15\" >0.82</td>\n",
       "      <td id=\"T_a9ae5_row6_col16\" class=\"data row6 col16\" >0.63</td>\n",
       "      <td id=\"T_a9ae5_row6_col17\" class=\"data row6 col17\" >0.58</td>\n",
       "      <td id=\"T_a9ae5_row6_col18\" class=\"data row6 col18\" >0.63</td>\n",
       "      <td id=\"T_a9ae5_row6_col19\" class=\"data row6 col19\" >0.65</td>\n",
       "      <td id=\"T_a9ae5_row6_col20\" class=\"data row6 col20\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row6_col21\" class=\"data row6 col21\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row6_col22\" class=\"data row6 col22\" >0.64</td>\n",
       "      <td id=\"T_a9ae5_row6_col23\" class=\"data row6 col23\" >0.65</td>\n",
       "      <td id=\"T_a9ae5_row6_col24\" class=\"data row6 col24\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row6_col25\" class=\"data row6 col25\" >0.66</td>\n",
       "      <td id=\"T_a9ae5_row6_col26\" class=\"data row6 col26\" >0.64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f3139302e30>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval = ts_100.evaluate()\n",
    "table = evaluation_as_table(eval)\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100_all = Collection.from_json(\"thoughtsource_100.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval = ts_100_all.evaluate()\n",
    "evaluation_as_table(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100 = ts_100.merge(ts_100_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100.dump(\"thoughtsource_100\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100 = unique_index_correction(ts_100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100.dump(\"thoughtsource_100_corrected_idx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import yaml file\n",
    "import yaml\n",
    "with open(\"/home/kon/work/ThoughtSource/notebooks/internal_documentation/problematic_items_TS100.yaml\", 'r') as file:\n",
    "    items = yaml.safe_load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': None,\n",
       " 'med_qa': ['454'],\n",
       " 'medmc_qa': ['9a29cd2c-e0d3-4d1e-82a7-32af10b47c9b'],\n",
       " 'open_book_qa': None,\n",
       " 'strategy_qa': ['cc266a3e29a3b8442b14'],\n",
       " 'worldtree': None}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "items[\"wrong\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/kon/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "empty = Collection.from_json(\"foo_empty\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty[\"commonsense_qa\"] = empty[\"med_qa\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty.dump(\"foo_empty_namechange\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Column mismatch between batch {'generated_cot', 'id', 'feedback'} and features {'ref_id', 'type', 'context', 'generated_cot', 'answer', 'question', 'feedback', 'id', 'cot', 'choices'}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m only_cot \u001b[39m=\u001b[39m Collection\u001b[39m.\u001b[39;49mfrom_json(\u001b[39m\"\u001b[39;49m\u001b[39mfoo_only_gen_cot\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
      "File \u001b[0;32m~/work/ThoughtSource/libs/cot/cot/dataloader.py:317\u001b[0m, in \u001b[0;36mCollection.from_json\u001b[0;34m(path_or_json, download_mode, source)\u001b[0m\n\u001b[1;32m    314\u001b[0m             \u001b[39mif\u001b[39;00m key \u001b[39min\u001b[39;00m dic:\n\u001b[1;32m    315\u001b[0m                 \u001b[39mdel\u001b[39;00m dic[key]\n\u001b[0;32m--> 317\u001b[0m         dataset_dict[split_name] \u001b[39m=\u001b[39m ds\u001b[39m.\u001b[39;49mDataset\u001b[39m.\u001b[39;49mfrom_dict(dic, info\u001b[39m.\u001b[39;49mfeatures, info, split)\n\u001b[1;32m    318\u001b[0m     collection[dataset_name] \u001b[39m=\u001b[39m ds\u001b[39m.\u001b[39mDatasetDict(dataset_dict)\n\u001b[1;32m    319\u001b[0m \u001b[39mreturn\u001b[39;00m collection\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:817\u001b[0m, in \u001b[0;36mDataset.from_dict\u001b[0;34m(cls, mapping, features, info, split)\u001b[0m\n\u001b[1;32m    815\u001b[0m info\u001b[39m.\u001b[39mfeatures \u001b[39m=\u001b[39m features\n\u001b[1;32m    816\u001b[0m \u001b[39mif\u001b[39;00m features \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 817\u001b[0m     mapping \u001b[39m=\u001b[39m features\u001b[39m.\u001b[39;49mencode_batch(mapping)\n\u001b[1;32m    818\u001b[0m mapping \u001b[39m=\u001b[39m {\n\u001b[1;32m    819\u001b[0m     col: OptimizedTypedSequence(data, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39mfeatures[col] \u001b[39mif\u001b[39;00m features \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m, col\u001b[39m=\u001b[39mcol)\n\u001b[1;32m    820\u001b[0m     \u001b[39mfor\u001b[39;00m col, data \u001b[39min\u001b[39;00m mapping\u001b[39m.\u001b[39mitems()\n\u001b[1;32m    821\u001b[0m }\n\u001b[1;32m    822\u001b[0m pa_table \u001b[39m=\u001b[39m InMemoryTable\u001b[39m.\u001b[39mfrom_pydict(mapping\u001b[39m=\u001b[39mmapping)\n",
      "File \u001b[0;32m~/work/ThoughtSource/venv/lib/python3.10/site-packages/datasets/features/features.py:1370\u001b[0m, in \u001b[0;36mFeatures.encode_batch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m   1368\u001b[0m encoded_batch \u001b[39m=\u001b[39m {}\n\u001b[1;32m   1369\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mset\u001b[39m(batch) \u001b[39m!=\u001b[39m \u001b[39mset\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m-> 1370\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mColumn mismatch between batch \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mset\u001b[39m(batch)\u001b[39m}\u001b[39;00m\u001b[39m and features \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mset\u001b[39m(\u001b[39mself\u001b[39m)\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m   1371\u001b[0m \u001b[39mfor\u001b[39;00m key, column \u001b[39min\u001b[39;00m batch\u001b[39m.\u001b[39mitems():\n\u001b[1;32m   1372\u001b[0m     column \u001b[39m=\u001b[39m cast_to_python_objects(column)\n",
      "\u001b[0;31mValueError\u001b[0m: Column mismatch between batch {'generated_cot', 'id', 'feedback'} and features {'ref_id', 'type', 'context', 'generated_cot', 'answer', 'question', 'feedback', 'id', 'cot', 'choices'}"
     ]
    }
   ],
   "source": [
    "only_cot = Collection.from_json(\"foo_only_gen_cot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "only_cot.select_generated_cots(instruction=\"qa-01\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "only_cot.dump(\"foo_only_gen_cot.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty.merge()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a19cb823e24c98ee05a9cfa4a3a579b5d56d4c1a735f2a12456750b95a1e155e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
